--- title: Establish Validity of approach keywords: fastai sidebar: home_sidebar summary: "We take a imagenet pretrained resnet18 model and train it on our class seperation task to see if the approach even works" ---
PATH = untar_data(URLs.IMAGEWOOF_320)
PATH.ls()
[PosixPath('/home/ubuntu/.fastai/data/imagewoof-320/val'),
 PosixPath('/home/ubuntu/.fastai/data/imagewoof-320/train')]
data = ImageDataBunch.from_folder(PATH, train='train', valid='val',
                                 ds_tfms=get_transforms(), size=224)

data.show_batch()

len(data.train_ds)
12454
len(data.valid_ds)
500
TRAIN = PATH/'train'
TRAIN.ls()
[PosixPath('/home/ubuntu/.fastai/data/imagewoof-320/train/n02093754'),
 PosixPath('/home/ubuntu/.fastai/data/imagewoof-320/train/n02088364'),
 PosixPath('/home/ubuntu/.fastai/data/imagewoof-320/train/n02089973'),
 PosixPath('/home/ubuntu/.fastai/data/imagewoof-320/train/n02087394'),
 PosixPath('/home/ubuntu/.fastai/data/imagewoof-320/train/n02111889'),
 PosixPath('/home/ubuntu/.fastai/data/imagewoof-320/train/n02099601'),
 PosixPath('/home/ubuntu/.fastai/data/imagewoof-320/train/n02105641'),
 PosixPath('/home/ubuntu/.fastai/data/imagewoof-320/train/n02086240'),
 PosixPath('/home/ubuntu/.fastai/data/imagewoof-320/train/n02115641'),
 PosixPath('/home/ubuntu/.fastai/data/imagewoof-320/train/n02096294')]

Make CSV

def get_label_from_names(names:Path):
    labels = []
    if is_listy(names):
        for name in names: 
            labels.append(Path(name).parent.name)
        return labels
    else: 
        return Path(names).parent.name
    
names = get_files(TRAIN, recurse=True, extensions='.JPEG')
get_label_from_names(names[:10])
['n02093754',
 'n02093754',
 'n02093754',
 'n02093754',
 'n02093754',
 'n02093754',
 'n02093754',
 'n02093754',
 'n02093754',
 'n02093754']
def make_df(names:list):
    names2 = names.copy()
    random.shuffle(names2)
    df = pd.DataFrame(data=[names, names2]).T
    labels1 = np.array(get_label_from_names(list(df.iloc[:,0])))
    labels2 = np.array(get_label_from_names(list(df.iloc[:,1])))
    labels = (labels1 == labels2).astype(np.int32)
    df['label'] = labels
    df.columns = ['image1', 'image2', 'label']
    return df

df = make_df(get_files(TRAIN, recurse=True, extensions='.JPEG'))
df.head()
image1 image2 label
0 /home/ubuntu/.fastai/data/imagewoof-320/train/... /home/ubuntu/.fastai/data/imagewoof-320/train/... 0
1 /home/ubuntu/.fastai/data/imagewoof-320/train/... /home/ubuntu/.fastai/data/imagewoof-320/train/... 0
2 /home/ubuntu/.fastai/data/imagewoof-320/train/... /home/ubuntu/.fastai/data/imagewoof-320/train/... 0
3 /home/ubuntu/.fastai/data/imagewoof-320/train/... /home/ubuntu/.fastai/data/imagewoof-320/train/... 0
4 /home/ubuntu/.fastai/data/imagewoof-320/train/... /home/ubuntu/.fastai/data/imagewoof-320/train/... 0
def make_csv_from_path(TRAIN=TRAIN, mult=5):
    TRAIN = Path(TRAIN)
    names60 = []
    dfs = []
    for d in TRAIN.ls():
        names = d.ls() * mult
        random.shuffle(names)
        names40 = names[:int(len(names)*0.4)]
        names60.extend(names[int(len(names)*0.4):])
        dfs.append(make_df(names40))
        
    df60 = make_df(names60)
    for df in dfs:
        df60 = df60.append(df)

    return df60

df = make_csv_from_path(TRAIN)
df.hist()
array([[<matplotlib.axes._subplots.AxesSubplot object at 0x7f14c7d4b790>]], dtype=object)

FastAI Databunch

df.head()
image1 image2 label
0 /home/ubuntu/.fastai/data/imagewoof-320/train/... /home/ubuntu/.fastai/data/imagewoof-320/train/... 0
1 /home/ubuntu/.fastai/data/imagewoof-320/train/... /home/ubuntu/.fastai/data/imagewoof-320/train/... 1
2 /home/ubuntu/.fastai/data/imagewoof-320/train/... /home/ubuntu/.fastai/data/imagewoof-320/train/... 0
3 /home/ubuntu/.fastai/data/imagewoof-320/train/... /home/ubuntu/.fastai/data/imagewoof-320/train/... 0
4 /home/ubuntu/.fastai/data/imagewoof-320/train/... /home/ubuntu/.fastai/data/imagewoof-320/train/... 0
class DoubleImageList(ImageList):
    def open(self, fn):
        im1 = open_image(fn[0]).resize(320)
        im2 = open_image(fn[1]).resize(320)
        black_space = torch.zeros(3, 320, 20)
        return Image(torch.cat([im1.data, black_space, im2.data], dim = 2))
data = (DoubleImageList.from_df(df, path='/', cols=['image1', 'image2'])
        .split_by_rand_pct(0.2)
        .label_from_df(cols=['label'])
        .transform(get_transforms())
        .databunch(bs=64)
        .normalize(imagenet_stats))
data.show_batch()

Model

predictions = None
preds = None
learn = None
gc.collect()
20
learn = cnn_learner(data, models.resnet18, metrics=[accuracy])
learn.fit_one_cycle(3)
epoch train_loss valid_loss accuracy time
0 0.406716 0.297715 0.877389 07:52
1 0.291941 0.212959 0.918982 07:53
2 0.245086 0.187337 0.931106 07:54
learn.path = Path.cwd()/'models'
learn.load('first')
Learner(data=ImageDataBunch;

Train: LabelList (49816 items)
x: DoubleImageList
Image (3, 320, 660),Image (3, 320, 660),Image (3, 320, 660),Image (3, 320, 660),Image (3, 320, 660)
y: CategoryList
0,0,0,0,0
Path: /;

Valid: LabelList (12454 items)
x: DoubleImageList
Image (3, 320, 660),Image (3, 320, 660),Image (3, 320, 660),Image (3, 320, 660),Image (3, 320, 660)
y: CategoryList
1,1,0,0,1
Path: /;

Test: None, model=Sequential(
  (0): Sequential(
    (0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)
    (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (2): ReLU(inplace=True)
    (3): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)
    (4): Sequential(
      (0): BasicBlock(
        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (1): BasicBlock(
        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
    )
    (5): Sequential(
      (0): BasicBlock(
        (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (downsample): Sequential(
          (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)
          (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        )
      )
      (1): BasicBlock(
        (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
    )
    (6): Sequential(
      (0): BasicBlock(
        (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (downsample): Sequential(
          (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)
          (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        )
      )
      (1): BasicBlock(
        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
    )
    (7): Sequential(
      (0): BasicBlock(
        (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (downsample): Sequential(
          (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)
          (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        )
      )
      (1): BasicBlock(
        (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
    )
  )
  (1): Sequential(
    (0): AdaptiveConcatPool2d(
      (ap): AdaptiveAvgPool2d(output_size=1)
      (mp): AdaptiveMaxPool2d(output_size=1)
    )
    (1): Flatten()
    (2): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (3): Dropout(p=0.25, inplace=False)
    (4): Linear(in_features=1024, out_features=512, bias=True)
    (5): ReLU(inplace=True)
    (6): BatchNorm1d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (7): Dropout(p=0.5, inplace=False)
    (8): Linear(in_features=512, out_features=2, bias=True)
  )
), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[<function accuracy at 0x7f14e0b1ff80>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('/home/ubuntu/akash/memory_net/models'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True, silent=False)], callbacks=[], layer_groups=[Sequential(
  (0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)
  (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (2): ReLU(inplace=True)
  (3): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)
  (4): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
  (5): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (6): ReLU(inplace=True)
  (7): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
  (8): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (9): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
  (10): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (11): ReLU(inplace=True)
  (12): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
  (13): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (14): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
  (15): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (16): ReLU(inplace=True)
  (17): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
  (18): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (19): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)
  (20): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (21): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
  (22): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (23): ReLU(inplace=True)
  (24): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
  (25): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
), Sequential(
  (0): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
  (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (2): ReLU(inplace=True)
  (3): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
  (4): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (5): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)
  (6): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (7): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
  (8): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (9): ReLU(inplace=True)
  (10): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
  (11): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (12): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
  (13): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (14): ReLU(inplace=True)
  (15): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
  (16): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (17): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)
  (18): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (19): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
  (20): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (21): ReLU(inplace=True)
  (22): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
  (23): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
), Sequential(
  (0): AdaptiveAvgPool2d(output_size=1)
  (1): AdaptiveMaxPool2d(output_size=1)
  (2): Flatten()
  (3): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (4): Dropout(p=0.25, inplace=False)
  (5): Linear(in_features=1024, out_features=512, bias=True)
  (6): ReLU(inplace=True)
  (7): BatchNorm1d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (8): Dropout(p=0.5, inplace=False)
  (9): Linear(in_features=512, out_features=2, bias=True)
)], add_time=True, silent=False)
learn.save('first')

Testing

  1. Take N images from each class as query images
  2. For each image in validation set:
    1. Create sample with each support image
    2. Stack in one batch
    3. Get predictions
    4. Get mean probabilities
    5. ARGMAX
TRAIN = TRAIN
VAL = PATH/'val'
num_images = 5
classes = 10
query_images = OrderedDict()
random.seed(42)
for d in TRAIN.ls():
    cls = d.name
    images = d.ls()
    random.shuffle(images)
    images = images[:num_images]
    query_images[cls] = images
samples = get_files(VAL, extensions='.JPEG', recurse=True)
def create_sample_from_paths(path1, path2):
    im1 = open_image(path1).resize(320)
    im2 = open_image(path2).resize(320)
    black_space = torch.zeros(3, 320, 20)
    image = torch.cat([im1.data, black_space, im2.data], dim = 2)
    image = learn.data.norm((image, torch.zeros(1)))[0].unsqueeze(0).cuda()
    return image
from tqdm import tqdm_notebook as tqdm
total, accurate, accuracy = 0.0, 0.0, 0.0
predictions = []
pbar = tqdm(samples)
learn.model.eval()
for sample in pbar:
    label = get_label_from_names(sample)
    positive, negative = [], []
    for key, value in query_images.items():
        if key == label:
            for val in value:
                positive.append(create_sample_from_paths(sample, val))
        else:
            for val in value:
                negative.append(create_sample_from_paths(sample, val))
    positive.extend(negative)
    stack = torch.cat(positive, dim=0)
    predictions.append(learn.model(stack)[:,1].reshape(classes, num_images).detach().cpu())
    preds = predictions[-1].mean(1).argmax().item()
    if preds == 0:
        accurate += 1
    total += 1
    accuracy = accurate / total
    pbar.set_description(f"Accuracy : {accuracy}")
print(accurate/total)
0.848
new_preds = torch.stack(predictions)
new_preds.shape
torch.Size([500, 10, 5])
np.save('predictions.npy', new_preds.numpy())
Path.cwd().ls()
[PosixPath('/home/ubuntu/akash/memory_net/LICENSE'),
 PosixPath('/home/ubuntu/akash/memory_net/setup.py'),
 PosixPath('/home/ubuntu/akash/memory_net/settings.ini'),
 PosixPath('/home/ubuntu/akash/memory_net/docs'),
 PosixPath('/home/ubuntu/akash/memory_net/.git'),
 PosixPath('/home/ubuntu/akash/memory_net/README.md'),
 PosixPath('/home/ubuntu/akash/memory_net/.gitattributes'),
 PosixPath('/home/ubuntu/akash/memory_net/predictions.npy'),
 PosixPath('/home/ubuntu/akash/memory_net/00_core.ipynb'),
 PosixPath('/home/ubuntu/akash/memory_net/.github'),
 PosixPath('/home/ubuntu/akash/memory_net/00_core-Copy1.ipynb'),
 PosixPath('/home/ubuntu/akash/memory_net/index.ipynb'),
 PosixPath('/home/ubuntu/akash/memory_net/CONTRIBUTING.md'),
 PosixPath('/home/ubuntu/akash/memory_net/.gitconfig'),
 PosixPath('/home/ubuntu/akash/memory_net/models'),
 PosixPath('/home/ubuntu/akash/memory_net/.gitignore'),
 PosixPath('/home/ubuntu/akash/memory_net/.ipynb_checkpoints')]